# coding:utf-8
from .data_collector import DataCollector
from bs4 import BeautifulSoup
import pandas as pd
import json


class NewsCrawler(DataCollector):
    news_type_id = "2510"
    news_limit_count = 50
    news_items = []

    def __init__(self):
        DataCollector.__init__(self)
        self.algorithm_name = "新闻采集"
        self.ipynb_template_name = "news_crawler-template.ipynb"

    def implent(self):
        DataCollector.implent(self)
        news_urls = []
        page = 1
        while True:
            url = "https://feed.mix.sina.com.cn/api/roll/get?pageid=153&lid=%s&k=&num=50&page=%d" % (self.news_type_id, page)
            result = self.http.request(method="GET", url=url)
            json_data = json.loads(result.data.decode())
            for item in json_data["result"]["data"]:
                news_urls.append(item["url"])
            if(len(news_urls) >= self.news_limit_count):
                break
            else:
                page = page + 1
                continue
        news_urls = news_urls[:self.news_limit_count]
        
        for news_url in news_urls:
            result = self.http.request(method="GET", url=news_url)
            soup = BeautifulSoup(result.data.decode(), "html.parser")
            if soup.find("h1", attrs={"class":"main-title"}):
                title = soup.find("h1", attrs={"class":"main-title"}).text.strip()
                publish_time = soup.find("span", attrs={"class":"date"}).text.strip()
                content = soup.find("div", attrs={"class":"article"}).text.strip()
                abstract = content[:300] + "……"
                self.news_items.append([title, publish_time, abstract, content, news_url])
            
        self.df = pd.DataFrame(self.news_items, columns=[
                               "新闻标题", "发布时间", "新闻摘要", "新闻内容" , "链接地址"])

    def prepareIpynbItems(self):
        DataCollector.prepareIpynbItems(self)
        self.ipynb_items["#news_type_id#"] = self.news_type_id
        self.ipynb_items["#news_limit_count#"] = self.news_limit_count
